package com.o2o.cleaning.month.platform.ebusiness_plat.nyb

import com.o2o.utils.Iargs
import org.apache.spark.sql.SparkSession

object CheckDataDetail {

  def main(args: Array[String]): Unit = {

    val spark = SparkSession.builder()
      .appName("CheckDataDetail")
      .config("spark.debug.maxToStringFields", "2000")
      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
      .config("spark.sql.caseSensitive", "true")
      .config("es.nodes", "192.168.1.29")
      .config("es.port", "9200")
      .config("cluster.name", "O2OElastic")
      .config("es.net.http.auth.user", "elastic")
      .config("es.net.http.auth.pass", "changeme")
      .master("local[*]")
      .getOrCreate()

    val sc = spark.sparkContext
    sc.hadoopConfiguration.set("fs.s3a.access.key", "GAO7EO9FWKPJ8WFCQDME")
    sc.hadoopConfiguration.set("fs.s3a.secret.key", "LZ0xaHBSYKHaJ9ECDbX9f7zin79UZkXfGoNapRPL")
    sc.hadoopConfiguration.set("fs.s3a.endpoint", "https://obs.cn-north-1.myhuaweicloud.com")
    sc.setLogLevel("WARN")

    val monthArr = Array(8, 9)
    //    val indexArr: Array[String] = Array("taobao", "tmall", "jd", "suning", "txd", "guomei", "kaola", "yunji", "dangdang", "miya", "jumei", "secoo", "womaiwang", "rongyigou", "beibeiapp", "intime", "palmlife", "rsmsh", "sifulan")
    val indexArr: Array[String] = Array("taobao", "tmall", "jd", "suning", "txd", "guomei", "kaola", "yunji", "dangdang", "miya", "secoo", "womaiwang", "rongyigou", "beibeiapp")
    val result = s"s3a://o2o-dataproces-group/zyf/nyb/result/tempn_jd/2020/5/"
    //        spark.read.orc(s"s3a://dws-data/zhongfang/nanzhuang/sourceData/2022/4/taobao_15/").registerTempTable("t1")//.printSchema()
    spark.read.json(s"s3a://o2o-dataproces-group/zyf/nyb/source/2021/12/tmall/").registerTempTable("t1")
    //        spark.read.orc(s"s3a://o2o-dataproces-group/zyf/nyb/result/jd/2020/12/").registerTempTable("t1")
    //    spark.read.orc(s"s3a://o2o-dataproces-group/zyf/nyb/result/tmall/2022/3/").where("thirdCategoryId = '100210305'").show(10)
    //        spark.read.option("header", true).csv(s"s3a://o2o-dataproces-group/zyf/test/7/").registerTempTable("t1")
    //    spark.read.option("header", true).csv("D://test").registerTempTable("t1")
    spark.sql(
      """
        |select * from t1 where title like '%平邑%金银花%'
        |""".stripMargin).show(20)
    //            spark.sql(
    //              """
    //                |select priceText from t1 order by priceText desc
    //                |""".stripMargin).show(false)
    //    spark.sql(
    //      """
    //        |select *
    //        |,case when fourthCategoryId is not null then fourthCategoryId
    //        |      when thirdCategoryId is not null then thirdCategoryId
    //        |      when secondCategoryId is not null then secondCategoryId
    //        |      when firstCategoryId is not null then firstCategoryId
    //        |      else null end standardLastCategoryId
    //        |from t1
    //        |""".stripMargin)
    //            .show(false)
    //      .coalesce(1).write.mode("overwrite").option("header", true).csv("D://test1")

    //      .show(10)
    //          .registerTempTable("t1")
    //            spark.sql(
    //              """
    //                |select * from t1 where good_id = '608546126634'
    //                |""".stripMargin)
    //          .show(false)
    //              .coalesce(1).write.mode("overwrite").json("D://test")

    //        spark.sql(
    //          """
    //            |select count(1),sum(sellcount),sum(salesamount) from t1
    //            |""".stripMargin)
    //          .show(false)
    //      .coalesce(1).write.option("header", true).csv(s"D:\\test")
    //    for (month <- monthArr) {
    //      for (pingtai <- indexArr) {
    //        val value = s"s3a://o2o-dataproces-group/zyf/nyb/source/2021/${month}/${pingtai}/"
    //
    //        spark.read.json(value).registerTempTable("t1")
    //        spark.sql(
    //          """
    //            |select title
    //            |,firstCategoryId
    //            |,secondCategoryId
    //            |,thirdCategoryId
    //            |,fourthCategoryId
    //            |from  t1 where title like '%黄桃%'
    //            |""".stripMargin)
    //          .coalesce(1).write.option("header", true).csv(s"D:\\test\\${month}\\${pingtai}")
    //          .show(false)
    //      }
    //    }
    //
  }

}
